The goals / steps of this project are the following:
# Import Packages
import glob # OS dependencies to get file system details
import cv2
#importing some useful packages
import pickle
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import numpy as np
from ipywidgets import interact, interactive, fixed
%matplotlib inline
def show_images(images, gray=None, divider = 2):
"""
This is an utility function to show multiple images with different colour maps
:param images - An images list
:param gray - A flag to set default value for matplotlib imshow colour map. If the image
shape is 2( i.e binary image) then cmap value will be "gray"
:return: Nothing
"""
rows = (len(images)+1)//divider
plt.figure(figsize=(16, 16))
for idx, img in enumerate(images):
plt.subplot(rows, divider, idx+1)
# if the image is binary then it'll be printed as grayscale, otherwise colour map
# will be ignored
plt.imshow(img, cmap="gray" if len(img.shape) == 2 else gray)
plt.xticks([])
plt.yticks([])
plt.show()
# Loading test images from test_image directory
camera_cal_imgs = [mpimg.imread(path) for path in glob.glob("camera_cal/*")]
# Visualize calibration images
show_images(camera_cal_imgs[:4], divider=4)
def grayscale(img, opencv_read=False):
"""
:param img:
:param opencv_read:
:return:
"""
if opencv_read:
return cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
return cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
# prepare objects points
objp = np.zeros((6*9, 3), np.float32)
objp[:,:2] = np.mgrid[0:9, 0:6].T.reshape(-1,2)
# Arrays to store object points and iamg points from all the images
objpoints = []
imgpoints = []
def find_and_draw_chessboard(img, idx,axs, pattern_size= (9,6)):
gray = grayscale(img)
# find the chessboard corners
ret, corners = cv2.findChessboardCorners(gray, pattern_size, None)
# if found, add object points, image points
if ret:
objpoints.append(objp)
imgpoints.append(corners)
# draw and display the corners
cv2.drawChessboardCorners(img, pattern_size, corners, ret)
axs[idx].axis('off')
axs[idx].imshow(img)
# Draw subplots dynamically
fig, axs = plt.subplots(5,4, figsize=(16, 16))
axs = axs.ravel()
for idx, img in enumerate(camera_cal_imgs):
find_and_draw_chessboard(img,idx,axs)
#cache an image to further reuse
sample_img = mpimg.imread("camera_cal/calibration1.jpg")
#cache image size to further reuse
img_size = sample_img.shape[:2]
# Do Camera calibration given objects' points and images' points
ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints,img_size, None, None)
# Save the Camera calibration results for later use
dist_pickle = {"mtx": mtx, "dist": dist}
pickle.dump( dist_pickle, open( "resources/calibration.p", "wb" ) )
def undistort(img, mtx, dist):
"""
:param img:
:param mtx:
:param dist:
:return:
"""
return cv2.undistort(img, mtx, dist,None, mtx)
def undistort_image(img, cmatrix, distc ):
"""
:param sample_img:
:param cmatrix:
:param distc:
:return:
"""
udistord_img = undistort(img, cmatrix, distc)
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(16,16))
ax1.imshow(img)
ax1.set_title('Original Image', fontsize=18)
ax2.imshow(udistord_img)
ax2.set_title('Undistorted Image', fontsize=18)
# this can be used
return udistord_img
res = undistort_image(sample_img, mtx,dist)
test_images = [mpimg.imread(path) for path in glob.glob("test_images/*")]
#undistord images
undistort_images = list(map(lambda img: undistort_image(img, mtx, dist), test_images))
def corners_unwarp(img, src, dst):
"""
:param img: input image
:param src: source
:param dst: destination
:return:
"""
M = cv2.getPerspectiveTransform(src,dst) # magnitute
Minv = cv2.getPerspectiveTransform(dst, src)
h,w = img.shape[:2]
warped = cv2.warpPerspective(img, M, (w,h), flags=cv2.INTER_LINEAR)
return warped, Minv, M
height, width = test_images[0].shape[:2]
# source points
p1 = (575, 465)
p2 = (705, 465)
p3 = (255, 685)
p4 = (1050, 685)
line_color = (0, 255, 0) # Green
# destination points
pd1 = (450, 0)
pd2 = (width - 450, 0)
pd3 = (450, height)
pd4 = (width - 450, height)
def draw_polygon_on_image(img, line_color=(0, 255,0)):
"""
:param img:
:return:
"""
cv2.line(img, p1, p2, line_color, 3)
cv2.line(img, p2, p4, line_color, 3)
cv2.line(img, p4, p3, line_color, 3)
cv2.line(img, p3, p1, line_color, 3)
return img
src_selected_images = list(map(lambda img: draw_polygon_on_image(img), test_images))
show_images(src_selected_images)
def visualize_warped_images(img, src, dst):
unwarped, _, _ = corners_unwarp(img, src, dst)
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(16,16))
img = draw_polygon_on_image(img)
ax1.imshow(img)
ax1.set_title('Undistorted Image', fontsize=18)
ax2.imshow(unwarped)
ax2.set_title('Unwarped Image', fontsize=18)
return unwarped
# define source and destination points for tranform
src = np.float32([p1, p2, p3, p4])
dst = np.float32([pd1, pd2, pd3, pd4])
#
warped_images = list(map(lambda img: visualize_warped_images(img, src, dst), undistort_images))
def extract_rgb_color_spaces(uwimg):
unwarp_R = uwimg[:, :, 0]
unwarp_G = uwimg[:, :, 1]
unwarp_B = uwimg[:, :, 2]
return unwarp_R,unwarp_G,unwarp_B
def extract_hsv_color_spaces(uwimg):
unwarp_HSV = cv2.cvtColor(uwimg, cv2.COLOR_RGB2HSV)
unwarp_H = unwarp_HSV[:, :, 0]
unwarp_S = unwarp_HSV[:, :, 1]
unwarp_V = unwarp_HSV[:, :, 2]
return unwarp_H,unwarp_S,unwarp_V
def extract_hsl_color_spaces(uwimg):
unwarp_HSL = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)
unwarp_HSL_H = unwarp_HSL[:, :, 0]
unwarp_HSL_S = unwarp_HSL[:, :, 1]
unwarp_HSL_L = unwarp_HSL[:, :, 2]
return unwarp_HSL_H,unwarp_HSL_S,unwarp_HSL_L
def extract_lab_color_spaces(uwimg):
unwarped_LAB = cv2.cvtColor(uwimg, cv2.COLOR_RGB2Lab)
unwarp_L = unwarped_LAB[:,:,0]
unwarp_A = unwarped_LAB[:,:,1]
unwarp_B = unwarped_LAB[:,:,2]
return unwarp_L, unwarp_A,unwarp_B
def apply_rgb_filter(unwarp_img):
#RGB
unwarp_R,unwarp_G ,unwarp_B = extract_rgb_color_spaces(unwarp_img)
fig, axs = plt.subplots(1, 3, figsize=(16, 16))
axs = axs.ravel()
axs[0].imshow(unwarp_R, cmap='gray')
axs[0].set_title('RGB R-channel', fontsize=12)
axs[1].imshow(unwarp_G, cmap='gray')
axs[1].set_title('RGB G-Channel', fontsize=12)
axs[2].imshow(unwarp_B, cmap='gray')
axs[2].set_title('RGB B-channel', fontsize=12)
def apply_hsv_filter(unwarp_img):
# HSV
unwarp_H,unwarp_S,unwarp_V = extract_hsv_color_spaces(unwarp_img)
fig, axs = plt.subplots(1, 3, figsize=(16, 16))
axs = axs.ravel()
axs[0].imshow(unwarp_H, cmap='gray')
axs[0].set_title('HSV H-Channel', fontsize=12)
axs[1].imshow(unwarp_S, cmap='gray')
axs[1].set_title('HSV S-channel', fontsize=12)
axs[2].imshow(unwarp_V, cmap='gray')
axs[2].set_title('HSV V-Channel', fontsize=12)
def apply_hsl_filter(unwarp_img):
# HSL
unwarp_HSL_H,unwarp_HSL_S,unwarp_HSL_L = extract_hsl_color_spaces(unwarp_img)
fig, axs = plt.subplots(1, 3, figsize=(16, 16))
axs = axs.ravel()
axs[0].imshow(unwarp_HSL_H, cmap='gray')
axs[0].set_title('HSL H-Channel', fontsize=12)
axs[1].imshow(unwarp_HSL_S, cmap='gray')
axs[1].set_title('HSL S-channel', fontsize=12)
axs[2].imshow(unwarp_HSL_L, cmap='gray')
axs[2].set_title('HSL V-Channel', fontsize=12)
def apply_lab_filter(unwarp_img):
# LAB
unwarp_L, unwarp_A,unwarp_B = extract_lab_color_spaces(unwarp_img)
fig, axs = plt.subplots(1, 3, figsize=(16, 16))
axs = axs.ravel()
axs[0].imshow(unwarp_L, cmap='gray')
axs[0].set_title('LAB L-Channel', fontsize=12)
axs[1].imshow(unwarp_A, cmap='gray')
axs[1].set_title('LAB A-channel', fontsize=12)
axs[2].imshow(unwarp_B, cmap='gray')
axs[2].set_title('LAB L-Channel', fontsize=12)
def apply_color_filter(unwarp_img):
apply_rgb_filter(unwarp_img)
apply_hsv_filter(unwarp_img)
apply_hsl_filter(unwarp_img)
apply_lab_filter(unwarp_img)
img = warped_images[1]
apply_color_filter(img)
def abs_sobel_thresh(gray, orient='x', thresh_min=0, thresh_max=255):
# Apply the following steps to img
# 1) Convert to grayscale
# gray = grayscale(img)
# 2) Take the derivative in x or y given orient = 'x' or 'y'
dx = 1 if orient=='x' else 0
dy = 1 if orient=='y' else 0
sobel = cv2.Sobel(gray, cv2.CV_64F,dx ,dy)
# 3) Take the absolute value of the derivative or gradient
abs_sobel = np.absolute(sobel)
# 4) Scale to 8-bit (0 - 255) then convert to type = np.uint8
scaled_sobel = np.uint8(255*abs_sobel/np.max(abs_sobel))
# 5) Create a mask of 1's where the scaled gradient magnitude
# is > thresh_min and < thresh_max
binary_sobel = np.zeros_like(scaled_sobel)
binary_sobel[(scaled_sobel >= thresh_min) & (scaled_sobel <= thresh_max)] = 1
return binary_sobel
def apply_sobel_threshold(img, gray, min_thresh, max_thresh):
"""
:param unwarp_img:
:param min_thresh:
:param max_thresh:
:return:
"""
abs_sobel = abs_sobel_thresh(gray, 'x', min_thresh, max_thresh)
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 16))
ax1.imshow(img)
ax1.set_title('Unwarped Image', fontsize=18)
ax2.imshow(abs_sobel, cmap='gray')
ax2.set_title('Sobel Absolute', fontsize=18)
plt.show()
min_thresh=50
max_thresh=190
sample_img = warped_images[1]
gray = grayscale(sample_img)
apply_sobel_threshold(sample_img,gray,min_thresh, max_thresh)
sample_img = warped_images[1]
gray_rgb,_,_ = extract_rgb_color_spaces(sample_img)
apply_sobel_threshold(sample_img,gray_rgb,min_thresh, max_thresh)
sample_img = warped_images[1]
_,_,gray_hsv = extract_hsv_color_spaces(sample_img)
apply_sobel_threshold(sample_img,gray_hsv,min_thresh, max_thresh)
sample_img = warped_images[1]
_,gray_hsl,_ = extract_hsl_color_spaces(sample_img)
apply_sobel_threshold(sample_img,gray_hsl,min_thresh, max_thresh)
sample_img = warped_images[1]
gray_lab,_,_ = extract_lab_color_spaces(sample_img)
apply_sobel_threshold(sample_img,gray_lab,min_thresh, max_thresh)
def mag_threshold(gray, sobel_kernel=3, mag_thresh=(0, 255)):
# Convert to grayscale
# gray = grayscale(img) # extract HSL L color scpace
# Take both Sobel x and y gradients
sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
sobely = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
# Calculate the gradient magnitude
gradmag = np.sqrt(sobelx**2 + sobely**2)
# Rescale to 8 bit
scale_factor = np.max(gradmag)/255
gradmag = (gradmag/scale_factor).astype(np.uint8)
# Create a binary image of ones where threshold is met, zeros otherwise
binary_output = np.zeros_like(gradmag)
binary_output[(gradmag >= mag_thresh[0]) & (gradmag <= mag_thresh[1])] = 1
# Return the binary image
return binary_output
def apply_sobel_mag_gradient(uwimg,gray,sobel_kernel, min_thresh, max_thresh):
sobel_mag = mag_threshold(gray, sobel_kernel,(min_thresh, max_thresh))
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(16,16))
ax1.imshow(uwimg)
ax1.set_title('Unwarped Image', fontsize=18)
ax2.imshow(sobel_mag, cmap='gray')
ax2.set_title('Sobel Magnitude', fontsize=18)
min_thresh=50
max_thresh=190
sobel_kernel = 15
sample_img = warped_images[1]
gray = grayscale(sample_img)
apply_sobel_mag_gradient(sample_img, gray, sobel_kernel, min_thresh, max_thresh)
sample_img = warped_images[1]
gray_rgb,_,_ = extract_rgb_color_spaces(sample_img)
apply_sobel_mag_gradient(sample_img, gray_rgb, sobel_kernel, min_thresh, max_thresh)
sample_img = warped_images[1]
_,_,gray_hsv = extract_hsv_color_spaces(sample_img)
apply_sobel_mag_gradient(sample_img, gray_hsv, sobel_kernel, min_thresh, max_thresh)
sample_img = warped_images[1]
_,gray_hsl,_ = extract_hsl_color_spaces(sample_img)
apply_sobel_mag_gradient(sample_img, gray_hsl, sobel_kernel, min_thresh, max_thresh)
sample_img = warped_images[1]
gray_lab,_,_ = extract_lab_color_spaces(sample_img)
apply_sobel_mag_gradient(sample_img, gray_lab, sobel_kernel, min_thresh, max_thresh)
# Define a function to threshold an image for a given range and Sobel kernel
def dir_threshold(gray, sobel_kernel=3, thresh=(0, np.pi/2)):
# Grayscale
# gray = grayscale(img) # extract HSL L color scpace
# Calculate the x and y gradients
sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
sobely = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
# Take the absolute value of the gradient direction,
# apply a threshold, and create a binary image result
absgraddir = np.arctan2(np.absolute(sobely), np.absolute(sobelx))
binary_output = np.ones_like(absgraddir)
binary_output[(absgraddir >= thresh[0]) & (absgraddir <= thresh[1])] = 0
# Return the binary image
return binary_output
def apply_sobel_dir_gradient(uwimg,gray,sobel_kernel, min_thresh, max_thresh):
sobel_mag = dir_threshold(gray, sobel_kernel,(min_thresh, max_thresh))
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(16,16))
ax1.imshow(uwimg)
ax1.set_title('Unwarped Image', fontsize=18)
ax2.imshow(sobel_mag, cmap='gray')
ax2.set_title('Sobel Direction', fontsize=18)
dir_min_thresh=0.3
dir_max_thresh=1.51
dir_sobel_kernel = 15
sample_img = warped_images[1]
gray = grayscale(sample_img)
apply_sobel_dir_gradient(sample_img, gray, dir_sobel_kernel, dir_min_thresh, dir_max_thresh)
sample_img = warped_images[1]
gray_rgb,_,_ = extract_rgb_color_spaces(sample_img)
apply_sobel_dir_gradient(sample_img, gray_rgb, dir_sobel_kernel, dir_min_thresh, dir_max_thresh)
sample_img = warped_images[1]
_,_,gray_hsv = extract_hsv_color_spaces(sample_img)
apply_sobel_dir_gradient(sample_img, gray_hsv, dir_sobel_kernel, dir_min_thresh, dir_max_thresh)
sample_img = warped_images[1]
_,gray_hsl,_ = extract_hsl_color_spaces(sample_img)
apply_sobel_dir_gradient(sample_img, gray_hsl, dir_sobel_kernel, dir_min_thresh, dir_max_thresh)
sample_img = warped_images[1]
gray_lab,_,_ = extract_lab_color_spaces(sample_img)
apply_sobel_dir_gradient(sample_img, gray_lab, dir_sobel_kernel, dir_min_thresh, dir_max_thresh)
def combine_thresholds(unwarp_img, gray, mag_kernel, mag_thresh, dir_thresh, dir_kernel ):
gradx = abs_sobel_thresh(gray, orient='x', thresh_min=mag_thresh[0], thresh_max=mag_thresh[1])
grady = abs_sobel_thresh(gray, orient='y', thresh_min=mag_thresh[0], thresh_max=mag_thresh[1])
mag_binary = mag_threshold(gray, sobel_kernel=mag_kernel, mag_thresh=mag_thresh)
dir_binary = dir_threshold(gray, sobel_kernel=dir_kernel, thresh=dir_thresh)
combined = np.zeros_like(dir_binary)
combined[((gradx == 1) & (grady == 1)) | ((mag_binary == 1) & (dir_binary == 1))] = 1
# Visualize sobel magnitude + direction threshold
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(16,16))
ax1.imshow(unwarp_img)
ax1.set_title('Unwarped Image', fontsize=18)
ax2.imshow(combined, cmap='gray')
ax2.set_title('Sobel Magnitude + Direction', fontsize=18)
dir_thresh= (0.3 ,1.51)
dir_sobel_kernel = 15
mag_thresh = (50, 190)
mg_sobel_kernel = 15
sample_img = warped_images[1]
gray = grayscale(sample_img)
combine_thresholds(sample_img, gray, mg_sobel_kernel,mag_thresh, dir_thresh, dir_sobel_kernel)
sample_img = warped_images[1]
gray_rgb,_,_ = extract_rgb_color_spaces(sample_img)
combine_thresholds(sample_img, gray_rgb, mg_sobel_kernel,mag_thresh, dir_thresh, dir_sobel_kernel)
sample_img = warped_images[1]
_,_,gray_hsv = extract_hsv_color_spaces(sample_img)
combine_thresholds(sample_img, gray_hsv, mg_sobel_kernel,mag_thresh, dir_thresh, dir_sobel_kernel)
sample_img = warped_images[1]
_,gray_hsl,_ = extract_hsl_color_spaces(sample_img)
combine_thresholds(sample_img, gray_hsl, mg_sobel_kernel,mag_thresh, dir_thresh, dir_sobel_kernel)
sample_img = warped_images[1]
gray_lab,_,_ = extract_lab_color_spaces(sample_img)
combine_thresholds(sample_img, gray_lab, mg_sobel_kernel,mag_thresh, dir_thresh, dir_sobel_kernel)
def nomalize_hsl(unwarped_img, thresh=(210, 255) ):
_,gray_hsl,_ = extract_hsl_color_spaces(unwarped_img)
gray_hsl = gray_hsl*(255/np.max(gray_hsl))
binary_output = np.zeros_like(gray_hsl)
binary_output[(gray_hsl > thresh[0]) & (gray_hsl <= thresh[1])] = 1
return binary_output
sample_img = warped_images[0]
hsl_s = nomalize_hsl(sample_img)
plt.imshow(hsl_s, cmap='gray')
<matplotlib.image.AxesImage at 0x7f55ac06f190>
def nomalize_lab(unwarped_img, thresh=(170, 255) ):
# 1) Convert to LAB color space
_,_,gray_lab = extract_lab_color_spaces(sample_img)
# don't normalize if there are no yellows in the image
if np.max(gray_lab) > 175:
lab_b = gray_lab*(255/np.max(gray_lab))
# 2) Apply a threshold to the L channel
binary_output = np.zeros_like(gray_lab)
binary_output[((gray_lab > thresh[0]) & (gray_lab <= thresh[1]))] = 1
# 3) Return a binary image of threshold result
return binary_output
sample_img = warped_images[0]
lab = nomalize_lab(sample_img)
plt.imshow(lab, cmap='gray')
<matplotlib.image.AxesImage at 0x7f55a4d1e430>
def combine_unwarped_img(unwarped_img):
hsl = nomalize_hsl(unwarped_img)
lab = nomalize_lab(unwarped_img)
combined = np.zeros_like(hsl)
combined[(hsl ==1) | (lab ==1)] = 1
return combined
sample_img = warped_images[0]
combined = combine_unwarped_img(sample_img)
plt.imshow(combined, cmap='gray')
<matplotlib.image.AxesImage at 0x7f55a4ea0610>
for img in warped_images:
comb = combine_unwarped_img(sample_img)
# Visualize sobel magnitude + direction threshold
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(16,16))
ax1.imshow(img)
ax1.set_title('Unwarped Image', fontsize=18)
ax2.imshow(comb, cmap='gray')
ax2.set_title('Sobel Magnitude + Direction', fontsize=18)
# Download IPython notebook as HTML file
import os
os.system('jupyter nbconvert --to html Advanced_Lane_Finding.ipynb')